from lxml import etree
import requests
import Sha1Util
from concurrent.futures import ThreadPoolExecutor
import 录入数据库方法

url_list = []  # 子页面 url
id_list = []  # id
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"

}


def diqu():

    for i in range(0, len(url_list)):  # 获取具体城市内容网页
        chile_url = url_list[i]
        page = requests.get(url=chile_url, headers=headers).text
        tree = etree.HTML(page)
        title_list = tree.xpath('//div[@class="content_page_box"]//a/@href')

        for j in range(1, len(title_list)):
            chile_url = url_list[i].rstrip("/") + title_list[j]
            page = requests.get(url=chile_url, headers=headers)

            page_text = page.text
            tree = etree.HTML(page_text)
            id = id_list[i]
            url = tree.xpath('//ul[@class="content_list_wrap"]//li//a/@href ')
            joy = tree.xpath('//ul[@class="content_list_wrap"]//li//a//text()')
            diqu = tree.xpath('//ul[@class="content_list_wrap"]//li/div[1]//span/@title')
            liulangrenshu = tree.xpath('//ul[@class="content_list_wrap"]//li/div[2]//span/@title')
            timedata = tree.xpath('//ul[@class="content_list_wrap"]//li//div[@class="left date"]/@title')
            # 新增数据
            page1=requests.get(url=chile_url, headers=headers)
            code = page1.status_code
            if code == 200:
                addHomeData(id, url, joy, diqu, liulangrenshu, timedata)
            else:
                print('404 Not Found')



def getziyemian():
    '''
    获取数据库url,id
    :return:
    '''
    sql = 'select url,id from 兼职猫城市'
    result = 录入数据库方法.query(sql)
    for el in result:
        url = el[0]
        url_list.append(url + "")
        id = el[1]
        id_list.append(id)
    return url_list, id_list


def createTable():
    '''
    建表
    :return:
    '''
    sql = '''
    create table if not exists 兼职猫城市下地区(
    id varchar(50) not null,
    joyid varchar(50) not null,
    child_url varchar(100) not null,
    joy varchar(50) not null,
    diqu varchar(15) not null,
    liulangrenshu varchar(10) not null,
    timedata varchar(15) not null,
    状态码 int(3)  DEFAULT 0,
    PRIMARY KEY (`child_url`)
    )
    '''
    录入数据库方法.createTable(sql=sql)


def addHomeData(id, url, joy, diqu, liulangrenshu, timedata):
    for i in range(0,len(url)):
        joyid = Sha1Util.jia_mi(diqu[i].rstrip("/") + (url_list[i]+url[i]))

        sql = '''insert into 兼职猫城市下地区(id,joyid,child_url,joy,diqu,liulangrenshu,timedata,状态码)
        values('%s','%s','%s','%s','%s','%s','%s','%s')
        ''' % (id, joyid, (url_list[i].rstrip("/")+url[i]), joy[i], diqu[i], liulangrenshu[i], timedata[i],0)
        录入数据库方法.modifyTable(sql)


if __name__ == '__main__':
    createTable()
    getziyemian()
    with ThreadPoolExecutor(3) as th:
        for i in range(1,len(url_list)):
            th.submit(diqu(),i)

